import json
import multiprocessing as mp

import requests

other_policy_list = []
other_policy_base = open("other_policy_base/dt=20201212/part-00000-899c55da-2798-48b7-9168-8515943591f4-c000.json",
                         "r",
                         encoding="utf-8")
for i in other_policy_base.readlines():
    other_policy_one = json.loads(i)
    other_policy_list.append(other_policy_one)
export = []


def c(message):
    try:
        res = requests.get(message["bbd_url"])
    except:
        return
    res.encoding = 'utf-8'
    message["bbd_html"] = res.text
    export.append(message)
    json.dump(export, open("other_policy.json", "w", encoding="utf-8"), ensure_ascii=False, indent=2)


pool = mp.Pool(mp.cpu_count())
pool.map(c, other_policy_list)
